package contentSpider;

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.sql.PreparedStatement;
import java.sql.Statement;
import java.util.Calendar;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import util.HtmlUnit.SpiderContentTxt;

public class ContentSpider {

	Log logger = LogFactory.getLog(ContentSpider.class);

	public void contentSpider() {
		// 两个静态变量都是延时，一个是抓取两篇文章间的延时长度另一个是出错后暂停的延时长度
		System.setProperty("LOG_DIR", "./");
		String titlePub = null;
		Connection conn = null;
		Connection connTag = null;
		Statement stmt = null;
		Statement stmtTag = null;
		PreparedStatement pstmt = null;
		Connection connGetSite = null;
		Statement stmtGetSite = null;
		java.sql.ResultSet rsGetSite = null;
		String contentString = null;
		SpiderContentTxt sct = new SpiderContentTxt();
		// 把程序包在一个大的循环里出错后重新进入一个新循环，继续运行不中断
		// 声明一个wyc对象
		// 初始化内容处理器
		ContentCapture cc = null;
		try {
			Class.forName("com.mysql.jdbc.Driver");

			// 抓取需要的参数哪去完毕后，查询tagurl表拿到需要抓取的网页链接插入到url表
			conn = DriverManager.getConnection("jdbc:mysql://" + dBHost + "/" + dBName + "?user=root&password=777888999a!&useUnicode=true&characterEncoding=utf8");
			connGetSite = DriverManager.getConnection("jdbc:mysql://" + dBHost + "/" + dBName + "?user=root&password=777888999a!&useUnicode=true&characterEncoding=utf8");
			connTag = DriverManager.getConnection("jdbc:mysql://" + dBHost + "/" + dBName + "?user=root&password=777888999a!&useUnicode=true&characterEncoding=utf8");
			stmt = conn.createStatement();
			stmtTag = connTag.createStatement();
			stmtGetSite = connGetSite.createStatement();
			// ZHHant zht = new ZHHant();
			// zht.setFyHost(fyHost);
			String tableName = "sec" + sourceNet;
			// 从url表中查询需要采集的链接
			rsGetSite = stmtGetSite.executeQuery("select id,url,subid from " + tableName + " where spider = 0");
			while (rsGetSite.next()) {
				stmtTag.executeUpdate("update " + tableName + " set spider = 1 where id = " + rsGetSite.getInt(1));
				if(sourceNet==6||sourceNet==7){
					cc = new ContentCaptureWenDa();
				}else{
					cc = new ContentCaptureBlog();
				}
				
				cc.setDelRex(delRex);
				cc.setTitleRex(titleRex);
				cc.setContentRex(contentRex);
//System.out.println(rsGetSite.getString("url"));
				int reConnectTime = 5;
				while (reConnectTime > 0) {
					contentString = sct.SpiderPageContent(rsGetSite.getString("url"));
//System.out.println(contentString);
					if (contentString.length() > 10) {
						reConnectTime = reConnectTime - 5;
					} else {
						reConnectTime--;
					}
				}
//System.out.println(contentString.length());
				// 抓取对象出错的话执行下面一段，直接跳过
				if (contentString.length()<10) {
					stmtTag.executeUpdate("update " + tableName + " set spider = 1 where id = " + rsGetSite.getInt(1));
					logger.info("内容抓取为空内容");
					cc = null;
					continue;
				}
				// 抓取正常的情况下，对抓取下的字符串进行处理
				else {
					// contentString、title处理，拿出有效的content和title
					// contnetPub和title是有效的发布出去的最终内容，其中content还经过伪原创处理
					// 调用ContentRex提取内容
					contentString = cc.getContent(contentString);
					titlePub = cc.getTitle();
					//System.out.println(titlePub);
					
					//System.out.println(contentString);
					
					// titlePub过长的截取
					// 如果title长度小于2或content长度小于500文章也不发布，方法是设置copy = -2
//System.out.println(contentString);
					if (titlePub.length() < 2 || contentString.length() < 500) {
						stmtTag.executeUpdate("update " + tableName + " set spider = 1 where id = " + rsGetSite.getInt(1));
						logger.info("222222" + dBName + "   " + tableName + (rsGetSite.getInt(1) + 1) + "个数据内容太少" + "222222" + Calendar.getInstance().getTime());
//System.out.println("内容太少");
						cc = null;
						continue;

					} else {
//System.out.println("ppppp");
						pstmt = conn.prepareStatement("insert into content (id,title,content,sourceNet,pageURL,copy,subid) values (?,?,?,?,?,?,?)");
						pstmt.setInt(1, 0);
						pstmt.setString(2, titlePub.toString());
						pstmt.setString(3, contentString.toString());
						pstmt.setInt(4, sourceNet);
						pstmt.setString(5, rsGetSite.getString("url"));
						pstmt.setInt(6, 0);
						pstmt.setInt(7, rsGetSite.getInt("subid"));
						pstmt.execute();
//System.out.println(titlePub);
					}
				}
				// 更新sw位
				stmtTag.executeUpdate("update " + tableName + " set spider = 1 where id = " + rsGetSite.getInt(1));
				logger.info("***" + dBName + "----" + tableName + (rsGetSite.getInt(1) + 1) + "个数据" + "*********" + Calendar.getInstance().getTime());
				cc = null;
			}
			// 因为程序包在一个大的死循环里，完成后只能用这个break跳出循环
			logger.info("---------抓取Content完成，下一步break出循环----------");

		} catch (ClassNotFoundException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (SQLException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}finally {
			try {
				if (stmtTag != null) {
					stmtTag.close();
					stmtTag = null;
				}
				if (stmt != null) {
					stmt.close();
					stmt = null;
				}
				if (conn != null) {
					conn.close();
					conn = null;
				}
				if (connTag != null) {
					connTag.close();
					connTag = null;
				}
				if (pstmt != null) {
					pstmt.close();
					pstmt = null;
				}
				if (connGetSite != null) {
					connGetSite.close();
					connGetSite = null;
				}
				if (stmtGetSite != null) {
					stmtGetSite.close();
					stmtGetSite = null;
				}
				if (rsGetSite != null) {
					rsGetSite.close();
					rsGetSite = null;
				}
				if(sct != null){
					sct = null;
				}
				if(cc != null){
					cc = null;
				}
			} catch (SQLException e) {
				e.printStackTrace();
			}
		}
	}

	String dBName = null;

	public void setdBName(String dBName) {
		this.dBName = dBName;
	}

	public void setdBHost(String dBHost) {
		this.dBHost = dBHost;
	}

	public void setSourceNet(int sourceNet) {
		this.sourceNet = sourceNet;
	}

	String dBHost = null;
	int sourceNet = 0;

	String contentRex = null;

	public void setContentRex(String contentRex) {
		this.contentRex = contentRex;
	}

	public void setDelRex(String delRex) {
		this.delRex = delRex;
	}

	public void setTitleRex(String titleRex) {
		this.titleRex = titleRex;
	}

	String delRex = null;
	String titleRex = null;

	String cookiesString = null;

	public void setCookiesString(String cookiesString) {
		this.cookiesString = cookiesString;
	}

	public void setProxyString(String proxyString) {
		this.proxyString = proxyString;
	}

	String proxyString = null;
}
